{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [1] Pandas 数据清洗 - test()\n",
    "# 很多数据集存在数据缺失、数据格式错误、错误数据或重复数据的情况，\n",
    "# 如果要对使数据分析更加准确，就需要对这些没有用的数据进行处理。\n",
    "# 几种常见的空值： n/a、 NA、 NaN、 —、 na\n",
    "# \n",
    "import pandas as pd\n",
    "\n",
    "''' \n",
    "如果我们要删除包含空字段的行，可以使用 dropna() 方法，语法格式如下：\n",
    "DataFrame.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)\n",
    "参数说明：\n",
    "axis    - 默认为 0, 表示逢空值剔除整行，如果设置参数 axis=1 表示逢空值去掉整列。\n",
    "how     - 默认为 'any' 如果一行（或一列）里任何一个数据有出现 NA 就去掉整行，如果设置 how='all' 一行（或列）都是 NA 才去掉这整行。\n",
    "thresh  - 设置需要多少非空值的数据才可以保留下来的。\n",
    "subset  - 设置想要检查的列。如果是多个列，可以使用列名的 list 作为参数。\n",
    "inplace - 如果设置 True, 将计算得到的值直接覆盖之前的值并返回 None, 修改的是源数据。\n",
    "'''\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "print(df['NUM_BEDROOMS'])\n",
    "print()\n",
    "# Pandas 默认把 n/a 和 NA 当作空数据\n",
    "print(df['NUM_BEDROOMS'].isnull())\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [2] Pandas 数据清洗 - dropna()\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "# Pandas 默认把 n/a 和 NA 当作空数据\n",
    "new_df = df.dropna()\n",
    "print(new_df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [3] 移除 ST_NUM 列中字段值为空的行：\n",
    "# \n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "df.dropna(subset=['ST_NUM'], inplace=True)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [4] 使用 fillna() 方法来替换一些空字段\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "df.fillna(12345, inplace=True)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [5] 使用 fillna() 方法来替换指定列一些空字段\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "df['PID'].fillna(12345, inplace=True)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [6] 使用 mean() 方法计算列的均值并替换空单元格：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "x = df[\"ST_NUM\"].mean()\n",
    "print(x)\n",
    "print()\n",
    "df[\"ST_NUM\"].fillna(x, inplace=True)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [7] 使用 median() 方法计算列的中位数并替换空单元格：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "x = df[\"ST_NUM\"].median()\n",
    "print(x)\n",
    "print()\n",
    "df[\"ST_NUM\"].fillna(x, inplace=True)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [8] 使用 mode() 方法计算列的众数(一组数据中出现次数最多的那个数)并替换空单元格：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/property-data.csv')\n",
    "print(df)\n",
    "print()\n",
    "x = df[\"ST_NUM\"].mode()\n",
    "print(x)\n",
    "print()\n",
    "df[\"ST_NUM\"].fillna(x, inplace=True)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [9] Pandas 清洗格式错误数据\n",
    "# 数据格式错误的单元格会使数据分析变得困难，甚至不可能。\n",
    "# 我们可以通过包含空单元格的行，或者将列中的所有单元格转换为相同格式的数据。\n",
    "# 以下实例会格式化日期：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "# 第三个日期格式错误\n",
    "data = {\n",
    "    \"Date\": ['2020/12/01', '2020/12/02', '20201226'],\n",
    "    \"duration\": [50, 40, 45]\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(data, index=[\"day1\", \"day2\", \"day3\"])\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "print(df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [10] Pandas 清洗格式错误数据\n",
    "# 数据格式错误的单元格会使数据分析变得困难，甚至不可能。\n",
    "# 我们可以通过包含空单元格的行，或者将列中的所有单元格转换为相同格式的数据。\n",
    "# 以下实例会格式化日期：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "# 年龄错误\n",
    "person = {\n",
    "    \"name\": ['Google', 'Runoob', 'Taobao'],\n",
    "    \"age\": [50, 40, 12345]    # 12345 年龄数据是错误的\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(person)\n",
    "df.loc[2, 'age'] = 30  # 修改数据\n",
    "print(df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [11] Pandas 清洗格式错误数据\n",
    "# 数据格式错误的单元格会使数据分析变得困难，甚至不可能。\n",
    "# 我们可以通过包含空单元格的行，或者将列中的所有单元格转换为相同格式的数据。\n",
    "# 以下实例会格式化日期：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "# 年龄错误\n",
    "person = {\n",
    "    \"name\": ['Google', 'Runoob', 'Taobao'],\n",
    "    \"age\": [50, 200, 12345]\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(person)\n",
    "\n",
    "# 将 age 大于 120 的设置为 120:\n",
    "for x in df.index:\n",
    "  if df.loc[x, \"age\"] > 120:\n",
    "    df.loc[x, \"age\"] = 120\n",
    "\n",
    "print(df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [12] Pandas 清洗格式错误数据\n",
    "# 数据格式错误的单元格会使数据分析变得困难，甚至不可能。\n",
    "# 我们可以通过包含空单元格的行，或者将列中的所有单元格转换为相同格式的数据。\n",
    "# 以下实例会格式化日期：\n",
    "#\n",
    "import pandas as pd\n",
    "\n",
    "# 年龄错误\n",
    "person = {\n",
    "    \"name\": ['Google', 'Runoob', 'Taobao'],\n",
    "    \"age\": [50, 200, 12345]\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(person)\n",
    "\n",
    "# 将 age 大于 120 的删除:\n",
    "for x in df.index:\n",
    "  if df.loc[x, \"age\"] > 120:\n",
    "    df.drop(x, inplace=True)\n",
    "\n",
    "print(df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [13] Pandas 清洗重复数据\n",
    "# 如果我们要清洗重复数据，可以使用 duplicated() 和 drop_duplicates() 方法。\n",
    "# 如果对应的数据是重复的，duplicated() 会返回 True，否则返回 False。\n",
    "# 不改变源数据\n",
    "# \n",
    "import pandas as pd\n",
    "\n",
    "person = {\n",
    "    \"name\": ['Google', 'Runoob', 'Runoob', 'Taobao'],\n",
    "    \"age\": [50, 40, 40, 23]\n",
    "}\n",
    "df = pd.DataFrame(person)\n",
    "\n",
    "print(df.duplicated())\n",
    "print(df.drop_duplicates())\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "cf18841ace8313d0bc088ca146c17a6c0040e82121d5cb75c0ea07172309253d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
